<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2014-2016 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Datasets Overview</title>

    <link rel="stylesheet" href="../../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../_static/doctools.js"></script>
    <script type="text/javascript" src="../../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../../index.html" />
    <link rel="up" title="Datasets" href="index.html" />
    <link rel="next" title="Tables" href="table.html" />
    <link rel="prev" title="Datasets" href="index.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: developer-manual -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><a href="../../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a>
      </li>
      <li class="toctree-l1"><b><a href="../../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../getting-started/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../getting-started/sandbox/index.html">CDAP Sandbox</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../getting-started/sandbox/zip.html">二进制 Zip 文件</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../getting-started/sandbox/zip.html#cdap-sandbox">启动和停止 CDAP Sandbox</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../getting-started/sandbox/virtual-machine.html">虚拟机镜像</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../getting-started/sandbox/docker.html">Docker 镜像</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../getting-started/quick-start.html">快速入门</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../getting-started/dev-env.html">搭建开发环境</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../getting-started/start-stop-cdap.html">启动和停止 CDAP</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../getting-started/building-apps.html">构建并运行应用</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../overview/index.html"> 概述</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../overview/anatomy.html"> 大数据应用剖析</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../overview/modes.html"> 模式和组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../overview/abstractions.html"> 核心概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../overview/interfaces.html"> 编程接口</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="../index.html"> 抽象概念</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../core.html"> Core Abstractions</a></li>
<li class="toctree-l2"><a class="reference internal" href="../applications.html"> Applications</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html"> Datasets</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#"> Overview</a></li>
<li class="toctree-l3"><a class="reference internal" href="table.html"> Table API</a></li>
<li class="toctree-l3"><a class="reference internal" href="fileset.html"> FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="partitioned-fileset.html"> Partitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="time-partitioned-fileset.html"> TimePartitioned FileSets</a></li>
<li class="toctree-l3"><a class="reference internal" href="system-custom.html"> System and Custom Datasets</a></li>
<li class="toctree-l3"><a class="reference internal" href="permissions.html"> Dataset Permissions</a></li>
<li class="toctree-l3"><a class="reference internal" href="cube.html"> Cube Dataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../mapreduce-programs.html"> MapReduce Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../plugins.html"> Plugins</a></li>
<li class="toctree-l2"><a class="reference internal" href="../schedules.html"> Schedules</a></li>
<li class="toctree-l2"><a class="reference internal" href="../secure-keys.html"> Secure Keys</a></li>
<li class="toctree-l2"><a class="reference internal" href="../services.html"> Services</a></li>
<li class="toctree-l2"><a class="reference internal" href="../spark-programs.html"> Spark Programs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../workers.html"> Workers</a></li>
<li class="toctree-l2"><a class="reference internal" href="../workflows.html"> Workflows</a></li>
<li class="toctree-l2"><a class="reference internal" href="../artifacts.html"> Artifacts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../program-lifecycle.html"> Program Lifecycle</a></li>
<li class="toctree-l2"><a class="reference internal" href="../namespaces.html"> Namespaces</a></li>
<li class="toctree-l2"><a class="reference internal" href="../transaction-system.html"> Transaction System</a></li>
<li class="toctree-l2"><a class="reference internal" href="../transactional-messaging-system.html"> Transactional Messaging System</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../metadata/index.html"> 元数据</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/system-metadata.html"> System Metadata</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/discovery-lineage.html"> Discovery and Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/field-lineage.html"> Field Level Lineage</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/audit-logging.html"> Audit Logging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/metadata-ui.html"> CDAP Metadata UI</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../metadata/programmatic-metadata.html"> Accessing metadata programmatically</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../pipelines/index.html"> 数据流管道</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/concepts-design.html"> Concepts and Design</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/getting-started.html"> Getting Started</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/studio.html"> CDAP Studio</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/creating-pipelines.html"> Creating Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/running-pipelines.html"> Running Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/plugin-management.html"> Plugin Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/plugins/index.html"> Plugin Reference</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="../../pipelines/plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/developing-pipelines.html"> Developing Pipelines</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/developing-plugins/index.html"> Developing Plugins</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/developing-plugins/plugin-basics.html">Plugin Basics</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/developing-plugins/creating-a-plugin.html">Creating a Plugin</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/developing-plugins/presentation-plugins.html">Plugin Presentation</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/developing-plugins/testing-plugins.html">Testing Plugins</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../pipelines/developing-plugins/packaging-plugins.html">Packaging Plugins</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../pipelines/how-cdap-pipelines-work.html"> How CDAP Pipelines Work</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../cloud-runtimes/index.html"> 云平台运行</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../cloud-runtimes/concepts/index.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../cloud-runtimes/provisioners/index.html"> Provisioners</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/provisioners/gcp-dataproc.html">Google Dataproc</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/provisioners/aws-emr.html">Amazon Elastic MapReduce</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/provisioners/remote-hadoop.html">Remote Hadoop</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../cloud-runtimes/profiles/index.html"> Profiles</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/profiles/creating-profiles.html">Creating Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/profiles/assigning-profiles.html">Assigning Profiles</a></li>
<li class="toctree-l3"><a class="reference internal" href="../../cloud-runtimes/profiles/admin-controls.html">Admin Controls</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../cloud-runtimes/example/index.html"> Example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../security/index.html"> 安全</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../security/client-authentication.html">Client Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../security/cdap-authentication-clients-java.html">CDAP Authentication Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../security/cdap-authentication-clients-python.html">CDAP Authentication Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../security/custom-authentication.html">Custom Authentication</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../security/authorization-extensions.html">Authorization Extensions</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../testing/index.html"> 测试和调试</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../testing/testing.html"> Testing a CDAP Application</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../testing/debugging.html"> Debugging</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../testing/troubleshooting.html"> Troubleshooting</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../ingesting-tools/index.html"> 数据融合</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../ingesting-tools/cdap-stream-clients-java.html">CDAP Stream Client for Java</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../ingesting-tools/cdap-stream-clients-python.html">CDAP Stream Client for Python</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../ingesting-tools/cdap-stream-clients-ruby.html">CDAP Stream Client for Ruby</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../ingesting-tools/cdap-flume.html">CDAP Flume</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../data-exploration/index.html"> 数据探索</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../data-exploration/filesets.html"> Fileset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../data-exploration/tables.html"> Table Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../data-exploration/object-mapped-tables.html"> ObjectMappedTable Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../data-exploration/custom-datasets.html"> Custom Dataset Exploration</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../data-exploration/hive-execution-engines.html"> Hive Execution Engines</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../advanced/index.html"> 高级主题</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../../advanced/application-logback.html"> Application Logback</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../advanced/best-practices.html"> Best Practices</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../advanced/class-loading.html"> Class Loading</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../advanced/configuring-resources.html"> Configuring Program Resources</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../advanced/program-retry-policies.html"> Program Retry Policies</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="datasets-overview">
<span id="id1"></span><h1>Datasets Overview<a class="headerlink" href="#datasets-overview" title="Permalink to this headline">🔗</a></h1>
<div class="section" id="introduction">
<h2>Introduction<a class="headerlink" href="#introduction" title="Permalink to this headline">🔗</a></h2>
<p>The core datasets of CDAP are <em>Tables</em> and <em>FileSets</em>:</p>
<ul class="simple">
<li>Unlike relational database systems, CDAP <em>Tables</em> are not organized into rows with a fixed schema.
They are optimized for efficient storage of semi-structured data, data with unknown or variable
schema, or sparse data.</li>
<li>CDAP <em>FileSets</em> provide an abstraction over the raw file system, and associate properties such as
the format or the schema with the files they contain. In addition, partitioned file sets
allow addressing files by their partition meta data, removing the need for applications to
be aware of actual file system locations.</li>
</ul>
<p>Other datasets are built on top of tables and file sets. A dataset can implement
specific semantics around a core dataset, such as a key/value Table or a
counter Table. A dataset can also combine multiple datasets to create a
complex data pattern. For example, an indexed Table can be implemented
by using one Table for the data and a second Table for the index of that data.</p>
<p>A number of useful datasets—we refer to them as system datasets—are
included with CDAP, including key/value tables, indexed tables and
time series. You can implement your own data patterns as custom
datasets, on top of any combination of core and system datasets.</p>
</div>
<div class="section" id="creating-a-dataset">
<h2>Creating a Dataset<a class="headerlink" href="#creating-a-dataset" title="Permalink to this headline">🔗</a></h2>
<p>You can create a dataset in CDAP using either the <span class="xref std std-ref">http-restful-api-dataset</span>, the
<span class="xref std std-ref">Command Line Interface</span>, or in an <a class="reference internal" href="../applications.html#applications"><span class="std std-ref">application specification.</span></a></p>
<p>You tell applications to create a dataset if it does not already
exist by declaring the dataset details in the application specification.
For example, to create a DataSet named <em>myCounters</em> of type
<a class="reference external" href="../../../reference-manual/javadocs/io/cdap/cdap/api/dataset/lib/KeyValueTable.html">KeyValueTable</a>, write:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
  <span class="n">createDataset</span><span class="p">(</span><span class="s">&quot;myCounters&quot;</span><span class="p">,</span> <span class="n">KeyValueTable</span><span class="p">.</span><span class="na">class</span><span class="p">);</span>
  <span class="p">...</span>
</pre></div>
</div>
<p>Names (<em>myCounters</em>) that start with an underscore (<code class="docutils literal notranslate"><span class="pre">_</span></code>) will not be visible in the home
page of the <span class="xref std std-ref">CDAP UI</span>, though they will be visible elsewhere in the CDAP UI.</p>
</div>
<div class="section" id="using-datasets-in-programs">
<span id="datasets-in-programs"></span><h2>Using Datasets in Programs<a class="headerlink" href="#using-datasets-in-programs" title="Permalink to this headline">🔗</a></h2>
<p>There are two ways to use a dataset in a program:</p>
<div class="section" id="static-instantiation">
<span id="static-dataset-instantiation"></span><h3>Static Instantiation<a class="headerlink" href="#static-instantiation" title="Permalink to this headline">🔗</a></h3>
<p>You can instruct the CDAP runtime system to inject the dataset into a class member
with the <code class="docutils literal notranslate"><span class="pre">&#64;UseDataSet</span></code> annotation:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">class</span> <span class="nc">MyFlowlet</span> <span class="kd">extends</span> <span class="n">AbstractFlowlet</span> <span class="p">{</span>
  <span class="nd">@UseDataSet</span><span class="p">(</span><span class="s">&quot;myCounters&quot;</span><span class="p">)</span>
  <span class="kd">private</span> <span class="n">KeyValueTable</span> <span class="n">counters</span><span class="p">;</span>
  <span class="p">...</span>
  <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">String</span> <span class="n">key</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">counters</span><span class="p">.</span><span class="na">increment</span><span class="p">(</span><span class="n">key</span><span class="p">.</span><span class="na">getBytes</span><span class="p">(),</span> <span class="mi">1L</span><span class="p">);</span>
  <span class="p">}</span>
</pre></div>
</div>
<p>When starting the program, the runtime system reads the dataset specification from the metadata store and injects
an instance of the dataset class into the application. This dataset will participate in every transaction that is
executed by the program. If the program is multi-threaded (for example, an HTTP service handler), CDAP will make
sure that every thread has its own instance of the dataset.</p>
</div>
<div class="section" id="dynamic-instantiation">
<span id="dynamic-dataset-instantiation"></span><h3>Dynamic Instantiation<a class="headerlink" href="#dynamic-instantiation" title="Permalink to this headline">🔗</a></h3>
<p>If you don’t know the name of the dataset at compile time (and hence you cannot use
static instantiation), or if you want to use a dataset only for a short time, you can dynamically
request an instance of the dataset through the program context:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">class</span> <span class="nc">MyFlowlet</span> <span class="kd">extends</span> <span class="n">AbstractFlowlet</span> <span class="p">{</span>
  <span class="p">...</span>
  <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">String</span> <span class="n">key</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">KeyValueTable</span> <span class="n">counters</span> <span class="o">=</span> <span class="n">getContext</span><span class="p">().</span><span class="na">getDataset</span><span class="p">(</span><span class="s">&quot;myCounters&quot;</span><span class="p">);</span>
    <span class="n">counters</span><span class="p">.</span><span class="na">increment</span><span class="p">(</span><span class="n">key</span><span class="p">.</span><span class="na">getBytes</span><span class="p">(),</span> <span class="mi">1L</span><span class="p">);</span>
  <span class="p">}</span>
</pre></div>
</div>
<p>This dataset is instantiated at runtime, in this case every time the method <code class="docutils literal notranslate"><span class="pre">process</span></code> is invoked. To reduce the
overhead of repeatedly instantiating the same dataset, the CDAP runtime system caches dynamic datasets internally.
By default, the cached instance of a dataset will not expire, and it will participate in all transactions initiated
by the program.</p>
<p>For convenience, if you know the dataset name at the time the program starts, you can store a reference to
the dataset in a member variable at that time (similar to static datasets, but assigned explicitly by you):</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">class</span> <span class="nc">MyFlowlet</span> <span class="kd">extends</span> <span class="n">AbstractFlowlet</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">KeyValueTable</span> <span class="n">counters</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">FlowletContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
    <span class="kd">super</span><span class="p">.</span><span class="na">initialize</span><span class="p">(</span><span class="n">context</span><span class="p">);</span>
    <span class="n">counters</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="na">getDataset</span><span class="p">(</span><span class="s">&quot;myCounters&quot;</span><span class="p">);</span>
  <span class="p">}</span>

  <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">String</span> <span class="n">key</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">counters</span><span class="p">.</span><span class="na">increment</span><span class="p">(</span><span class="n">key</span><span class="p">.</span><span class="na">getBytes</span><span class="p">(),</span> <span class="mi">1L</span><span class="p">);</span>
  <span class="p">}</span>
</pre></div>
</div>
<p>Contrary to static datasets, dynamic datasets allow the release of the resources held by their Java classes
after you are finished using them. You can do that by calling the <code class="docutils literal notranslate"><span class="pre">discardDataset()</span></code> method of the program context:
it marks the dataset to be closed and removed from all transactions. If a transaction is
currently in progress, however, this will not happen until after the current transaction
is complete, because the discarded dataset may have performed data operations and
therefore still needs to participate in the commit (or rollback) of the transaction.</p>
<p>Discarding a dataset is useful:</p>
<ul class="simple">
<li>To ensure that the dataset is closed and its resources are released, as soon as a program does not need the dataset
any longer.</li>
<li>To refresh a dataset after its properties have been updated. Without discarding the dataset, the program would keep
the dataset in its cache and never pick up a change in the dataset’s properties. Discarding the dataset
ensures that it is removed from the cache after the current transaction. Therefore, the next time this dataset is
obtained using <code class="docutils literal notranslate"><span class="pre">getDataset()</span></code> in a subsequent transaction, it is guaranteed to return a fresh instance of the
dataset, hence picking up any properties that have changed since the program started.</li>
</ul>
<p>It is important to know that after discarding a dataset, it remains in the cache for the duration of the current
transaction. Be aware that if you call <code class="docutils literal notranslate"><span class="pre">getDataset()</span></code> again for the same dataset and arguments before the
transaction is complete, then that reverses the effect of discarding. It is therefore a good practice to discard
a dataset at the end of a transaction.</p>
<p>Discarding a dataset has the effect of releasing the dataset as soon as possible. If there
is no transaction, that is immediately. If there is a current transaction, that is as soon
as the transaction finishes.</p>
<p>Similarly to static datasets, if a program is multi-threaded, CDAP will make
sure that every thread has its own instance of each dynamic dataset—and in order to discard a dataset
from the cache, every thread that uses it must individually call <code class="docutils literal notranslate"><span class="pre">discardDataset()</span></code>.</p>
</div>
<div class="section" id="multi-threading-and-dataset-access">
<span id="multi-threading-dataset-access"></span><h3>Multi-threading and Dataset Access<a class="headerlink" href="#multi-threading-and-dataset-access" title="Permalink to this headline">🔗</a></h3>
<p>As mentioned above—under <a class="reference internal" href="#static-dataset-instantiation"><span class="std std-ref">static</span></a> and
<a class="reference internal" href="#dynamic-dataset-instantiation"><span class="std std-ref">dynamic</span></a> instantiation—if a program is
multi-threaded, CDAP will make sure that every thread has its own instance of a dataset.
This is because <strong>datasets are not thread-safe</strong>, cannot be shared across threads, and
each thread must operate on its own instance of a Dataset.</p>
<p>As a consequence, multiple threads accessing the same dataset will have different
instances of the same dataset.</p>
<p>As <a class="reference internal" href="../transaction-system.html#transaction-system"><span class="std std-ref">transactions</span></a> are not thread-safe, the dataset context of a
transaction as well as datasets obtained through it may not be shared across threads.</p>
</div>
<div class="section" id="cross-namespace-dataset-access">
<span id="id2"></span><h3>Cross-namespace Dataset Access<a class="headerlink" href="#cross-namespace-dataset-access" title="Permalink to this headline">🔗</a></h3>
<p>The dataset usage methods described above allow accessing datasets from the same <a class="reference internal" href="../namespaces.html#namespaces"><span class="std std-ref">namespace</span></a> in
which the program exists. However, <a class="reference internal" href="#dynamic-dataset-instantiation"><span class="std std-ref">Dynamic Dataset Instantiation</span></a> also allows
users to access datasets from a different namespace than the one in which the program accessing the dataset is running.
Typically, this may be required in scenarios where datasets are large enough to warrant sharing across namespaces, as
opposed to every namespace having its own copy. To use a dataset from a different namespace, users can pass a
<code class="docutils literal notranslate"><span class="pre">namespace</span></code> parameter to <code class="docutils literal notranslate"><span class="pre">getDataset()</span></code>:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">class</span> <span class="nc">MyFlowlet</span> <span class="kd">extends</span> <span class="n">AbstractFlowlet</span> <span class="p">{</span>

  <span class="kd">private</span> <span class="n">KeyValueTable</span> <span class="n">counters</span><span class="p">;</span>

  <span class="nd">@Override</span>
  <span class="kd">public</span> <span class="kt">void</span> <span class="nf">initialize</span><span class="p">(</span><span class="n">FlowletContext</span> <span class="n">context</span><span class="p">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="p">{</span>
    <span class="kd">super</span><span class="p">.</span><span class="na">initialize</span><span class="p">(</span><span class="n">context</span><span class="p">);</span>
    <span class="n">counters</span> <span class="o">=</span> <span class="n">context</span><span class="p">.</span><span class="na">getDataset</span><span class="p">(</span><span class="s">&quot;otherNamespace&quot;</span><span class="p">,</span> <span class="s">&quot;myCounters&quot;</span><span class="p">);</span>
  <span class="p">}</span>

  <span class="kt">void</span> <span class="nf">process</span><span class="p">(</span><span class="n">String</span> <span class="n">key</span><span class="p">)</span> <span class="p">{</span>
    <span class="n">counters</span><span class="p">.</span><span class="na">get</span><span class="p">(</span><span class="n">key</span><span class="p">.</span><span class="na">getBytes</span><span class="p">());</span>
  <span class="p">}</span>
</pre></div>
</div>
<p>Using this API, users can both read and write to a dataset in a different namespace.</p>
<p>Cross namespace access is not supported using <a class="reference internal" href="#static-dataset-instantiation"><span class="std std-ref">Static Dataset Instantiation</span></a>, since
doing so would require users to know the namespace at the time of development of the application.</p>
<p><strong>Note</strong>: On clusters with <span class="xref std std-ref">authorization</span> enabled, please refer to
<span class="xref std std-ref">authorization policy pushdown</span> for additional instructions.</p>
</div>
</div>
<div class="section" id="dataset-management-in-programs">
<span id="dataset-admin-in-programs"></span><h2>Dataset Management in Programs<a class="headerlink" href="#dataset-management-in-programs" title="Permalink to this headline">🔗</a></h2>
<p>Instantiating a dataset in a program allows you to perform any of the dataset’s data operations—the Java
methods defined in the dataset’s API. However, you cannot perform administrative operations such as creating or
dropping a dataset. For these operations, the program context offers an <code class="docutils literal notranslate"><span class="pre">Admin</span></code> interface that can be obtained
through the <code class="docutils literal notranslate"><span class="pre">getAdmin()</span></code> method of the context. This is available in all types of programs.</p>
<p>The handler defines several endpoints for dataset management, one of which can be used to create a new file set,
either by cloning an existing file set’s dataset properties, or by using the properties submitted in the request body.</p>
<p>Note that even though you can call dataset management methods within a transaction, these operations are <em>not</em>
transactional, and they are not rolled back in case the current transaction fails. It is advisable not to mix data
operations and dataset management operations within the same transaction.</p>
</div>
<div class="section" id="dataset-time-to-live-ttl">
<h2>Dataset Time-To-Live (TTL)<a class="headerlink" href="#dataset-time-to-live-ttl" title="Permalink to this headline">🔗</a></h2>
<p>Datasets can have a Time-To-Live (TTL) property that
governs how long data will be persisted in a specific dataset. TTL is configured as the
maximum age (in seconds) that data should be retained.</p>
<p>When you create a dataset, you can configure its TTL as part of the creation:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">configure</span><span class="p">()</span> <span class="p">{</span>
  <span class="n">createDataset</span><span class="p">(</span><span class="s">&quot;myCounters&quot;</span><span class="p">,</span> <span class="n">Table</span><span class="p">.</span><span class="na">class</span><span class="p">,</span>
                <span class="n">DatasetProperties</span><span class="p">.</span><span class="na">builder</span><span class="p">().</span><span class="na">add</span><span class="p">(</span><span class="n">Table</span><span class="p">.</span><span class="na">PROPERTY_TTL</span><span class="p">,</span>
                                                <span class="s">&quot;&lt;age in seconds&gt;&quot;</span><span class="p">).</span><span class="na">build</span><span class="p">());</span>
  <span class="p">...</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The default TTL for all datasets is infinite, meaning that data will never expire. The TTL
property of an existing dataset can be changed using the <span class="xref std std-ref">http-restful-api-dataset</span>.</p>
</div>
<div class="section" id="types-of-datasets">
<span id="datasets-overview-types"></span><h2>Types of Datasets<a class="headerlink" href="#types-of-datasets" title="Permalink to this headline">🔗</a></h2>
<p>A dataset abstraction is defined by a Java class that implements the <code class="docutils literal notranslate"><span class="pre">DatasetDefinition</span></code> interface.
The implementation of a dataset typically relies on one or more underlying (embedded) datasets.
For example, the <code class="docutils literal notranslate"><span class="pre">IndexedTable</span></code> dataset can be implemented by two underlying Table datasets—one holding the data and one holding the index.</p>
<p>We distinguish three categories of datasets: <em>core</em>, <em>system</em>, and <em>custom</em> datasets:</p>
<ul class="simple">
<li>The <a class="reference external" href="index.html#core-datasets"><strong>core</strong></a> datasets of the CDAP are Table and FileSet. Their implementations may use internal
CDAP classes hidden from developers.</li>
<li>A <a class="reference external" href="system-custom.html#system-datasets"><strong>system</strong></a> dataset is bundled with the CDAP and is built around
one or more underlying core or system datasets to implement a specific data pattern.</li>
<li>A <a class="reference external" href="system-custom.html#custom-datasets"><strong>custom</strong></a> dataset is implemented by you and can have arbitrary code and methods.
It is typically built around one or more Tables, FileSets (or other datasets)
to implement a specific data pattern.</li>
</ul>
<p>Each dataset is associated with exactly one dataset implementation to
manipulate it. Every dataset has a unique name and metadata that defines its behavior.
For example, every <code class="docutils literal notranslate"><span class="pre">IndexedTable</span></code> has a name and indexes a particular column of its primary table:
the name of that column is a metadata property of each dataset of this type.</p>
<div class="section" id="core-datasets">
<span id="id3"></span><h3>Core Datasets<a class="headerlink" href="#core-datasets" title="Permalink to this headline">🔗</a></h3>
<p><strong>Tables</strong> and <strong>FileSets</strong> are the core datasets,
and all other datasets are built using combinations of Tables and FileSets.</p>
<p>While these Tables have rows and columns similar to relational database tables, there are key differences:</p>
<ul class="simple">
<li>Tables have no fixed schema. Unlike relational database tables where every
row has the same schema, every row of a Table can have a different set of columns.</li>
<li>Because the set of columns is not known ahead of time, the columns of
a row do not have a rich type. All column values are byte arrays and
it is up to the application to convert them to and from rich types.
The column names and the row key are also byte arrays.</li>
<li>When reading from a Table, one need not know the names of the columns:
The read operation returns a map from column name to column value.
It is, however, possible to specify exactly which columns to read.</li>
<li>Tables are organized in a way that the columns of a row can be read
and written independently of other columns, and columns are ordered
in byte-lexicographic order. They are also known as <em>Ordered Columnar Tables</em>.</li>
</ul>
<p>A <a class="reference external" href="fileset.html"><strong>FileSet</strong></a> represents a collection of files in the file system that share some common attributes
such as the format and schema, while abstracting from the actual underlying file system interfaces.</p>
<ul>
<li><p class="first">Every file in a FileSet is in a location relative to the FileSet’s base directory.</p>
</li>
<li><p class="first">Knowing a file’s relative path, any program can obtain a <code class="docutils literal notranslate"><span class="pre">Location</span></code> for that file through a method
of the FileSet dataset. It can then interact directly with the file’s Location; for example, to write
data to the Location, or to read data from it.</p>
</li>
<li><p class="first">A FileSet can be used as the input or output to MapReduce. The MapReduce program need not specify
the input and output format to use, or configuration for these—the FileSet dataset provides this
information to the MapReduce runtime system.</p>
</li>
<li><p class="first">An abstraction of FileSets, <code class="docutils literal notranslate"><span class="pre">PartitionedFileSets</span></code> allow the associating of meta data (partitioning keys)
with each file. The file can then be addressed through its meta data, removing the need for programs to
be aware of actual file paths.</p>
</li>
<li><p class="first">A <code class="docutils literal notranslate"><span class="pre">TimePartitionedFileSet</span></code> is a further variation that uses a timestamp as the partitioning key.
Though it is not required that data in each partition be organized by time,
each partition is assigned a logical time.</p>
<p>This is in contrast to a <span class="xref std std-ref">Timeseries Table</span> dataset, where
time is the primary means of how data is organized, and both the data model and the
schema that represents the data are optimized for querying and aggregating over time
ranges.</p>
<p>Time-partitioned FileSets are typically written in batch: into large files, every <em>N</em> minutes or
hours…while a timeseries table is typically written in real-time, one data point at a
time.</p>
</li>
<li><p class="first">A <code class="docutils literal notranslate"><span class="pre">CubeDataset</span></code> is a multidimensional dataset, optimized for data warehousing and OLAP
(Online Analytical Processing) applications. A Cube dataset stores multidimensional facts,
provides a querying interface for retrieval of data and allows exploring of the stored data.</p>
<p>See <a class="reference internal" href="cube.html#datasets-cube"><span class="std std-ref">Cube Dataset</span></a> for: details on configuring a Cube dataset;
writing to and reading from it; and querying and exploring the data in a cube.</p>
<p>An example of using the Cube dataset is provided in the <span class="xref std std-ref">Data Analysis with OLAP
Cube</span> guide.</p>
</li>
</ul>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="Datasets" href="index.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Tables" href="table.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../../_static/js/js.cookie.js"></script><script type="text/javascript" src="../../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>